From 949f4e58ad0d0c78dd6ec8c10e9d1ba3dd6520bd Mon Sep 17 00:00:00 2001 From: "kaf24@scramble.cl.cam.ac.uk" Date: Mon, 24 Nov 2003 20:18:29 +0000 Subject: [PATCH] bitkeeper revision 1.634 (3fc267950JCnZdSjqz12f7QhAx9gWA) Many files: Cleanups to page reference counting in Xen. --- xen/arch/i386/mm.c | 6 +++--- xen/common/dom0_ops.c | 2 +- xen/common/dom_mem_ops.c | 10 ++++++---- xen/common/domain.c | 18 +++++++++++------- xen/common/memory.c | 34 +++++++++++++++------------------- xen/drivers/block/xen_block.c | 4 ++-- xen/include/xeno/mm.h | 29 +++++++++++++++++------------ xen/net/dev.c | 14 +++++++++----- xen/net/skbuff.c | 4 +++- 9 files changed, 67 insertions(+), 54 deletions(-) diff --git a/xen/arch/i386/mm.c b/xen/arch/i386/mm.c index a51ac43a23..5df703de7a 100644 --- a/xen/arch/i386/mm.c +++ b/xen/arch/i386/mm.c @@ -232,7 +232,7 @@ long set_gdt(struct task_struct *p, if ( (page->flags & PG_type_mask) != PGT_gdt_page ) { - if ( page->type_count != 0 ) + if ( page_type_count(page) != 0 ) goto out; /* Check all potential GDT entries in the page. */ @@ -253,7 +253,7 @@ long set_gdt(struct task_struct *p, page = frame_table + pfn; ASSERT((page->flags & PG_type_mask) == PGT_gdt_page); ASSERT((page->flags & PG_domain_mask) == p->domain); - ASSERT((page->type_count != 0) && (page->tot_count != 0)); + ASSERT((page_type_count(page) != 0) && (page_tot_count(page) != 0)); put_page_type(page); put_page_tot(page); } @@ -340,7 +340,7 @@ long do_update_descriptor( case PGT_writeable_page: break; default: - if ( page->type_count != 0 ) + if ( page_type_count(page) != 0 ) goto out; } diff --git a/xen/common/dom0_ops.c b/xen/common/dom0_ops.c index 8db4b5fedb..344a65f83a 100644 --- a/xen/common/dom0_ops.c +++ b/xen/common/dom0_ops.c @@ -344,7 +344,7 @@ long do_dom0_op(dom0_op_t *u_dom0_op) op.u.getpageframeinfo.domain = page->flags & PG_domain_mask; op.u.getpageframeinfo.type = NONE; - if ( page->type_count != 0 ) + if ( page_type_count(page) != 0 ) { switch ( page->flags & PG_type_mask ) { diff --git a/xen/common/dom_mem_ops.c b/xen/common/dom_mem_ops.c index 2e8696c533..c8869882ae 100644 --- a/xen/common/dom_mem_ops.c +++ b/xen/common/dom_mem_ops.c @@ -57,7 +57,8 @@ static long alloc_dom_mem(struct task_struct *p, reservation_increase_t op) /* Get a free page and add it to the domain's page list. */ pf = list_entry(temp, struct pfn_info, list); pf->flags |= p->domain; - pf->type_count = pf->tot_count = 0; + set_page_type_count(pf, 0); + set_page_tot_count(pf, 0); temp = temp->next; list_del(&pf->list); list_add_tail(&pf->list, &p->pg_head); @@ -109,12 +110,13 @@ static long free_dom_mem(struct task_struct *p, reservation_decrease_t op) } pf = &frame_table[mpfn]; - if ( (pf->type_count != 0) || - (pf->tot_count != 0) || + if ( (page_type_count(pf) != 0) || + (page_tot_count(pf) != 0) || ((pf->flags & PG_domain_mask) != p->domain) ) { DPRINTK("Bad page free for domain %d (%ld, %ld, %08lx)\n", - p->domain, pf->type_count, pf->tot_count, pf->flags); + p->domain, page_type_count(pf), + page_tot_count(pf), pf->flags); rc = -EINVAL; goto out; } diff --git a/xen/common/domain.c b/xen/common/domain.c index 1c1a25819b..eae232206b 100644 --- a/xen/common/domain.c +++ b/xen/common/domain.c @@ -245,7 +245,8 @@ unsigned int alloc_new_dom_mem(struct task_struct *p, unsigned int kbytes) { pf = list_entry(temp, struct pfn_info, list); pf->flags = p->domain; - pf->type_count = pf->tot_count = 0; + set_page_type_count(pf, 0); + set_page_tot_count(pf, 0); temp = temp->next; list_del(&pf->list); list_add_tail(&pf->list, &p->pg_head); @@ -273,7 +274,9 @@ void free_all_dom_mem(struct task_struct *p) while ( (ent = p->pg_head.next) != &p->pg_head ) { struct pfn_info *pf = list_entry(ent, struct pfn_info, list); - pf->type_count = pf->tot_count = pf->flags = 0; + set_page_type_count(pf, 0); + set_page_tot_count(pf, 0); + pf->flags = 0; ASSERT(ent->next->prev == ent); ASSERT(ent->prev->next == ent); list_del(ent); @@ -513,7 +516,8 @@ int setup_guestos(struct task_struct *p, dom0_createdomain_t *params, page = frame_table + (cur_address >> PAGE_SHIFT); page->flags = dom | PGT_writeable_page | PG_need_flush; - page->type_count = page->tot_count = 1; + set_page_type_count(page, 1); + set_page_tot_count(page, 1); /* Set up the MPT entry. */ machine_to_phys_mapping[cur_address >> PAGE_SHIFT] = count; @@ -535,7 +539,7 @@ int setup_guestos(struct task_struct *p, dom0_createdomain_t *params, *l1tab = mk_l1_pgentry(l1_pgentry_val(*l1tab) & ~_PAGE_RW); page = frame_table + l1_pgentry_to_pagenr(*l1tab); page->flags = dom | PGT_l1_page_table; - page->tot_count++; + get_page_tot(page); l1tab++; if( !((unsigned long)l1tab & (PAGE_SIZE - 1)) ) { @@ -544,9 +548,9 @@ int setup_guestos(struct task_struct *p, dom0_createdomain_t *params, l2tab++; } } - page->type_count |= REFCNT_PIN_BIT; - page->tot_count |= REFCNT_PIN_BIT; - page->flags = dom | PGT_l2_page_table; + get_page_type(page); /* guest_pinned */ + get_page_tot(page); /* guest_pinned */ + page->flags = dom | PG_guest_pinned | PGT_l2_page_table; unmap_domain_mem(l1start); /* Set up shared info area. */ diff --git a/xen/common/memory.c b/xen/common/memory.c index 01c846542a..77fe5822a2 100644 --- a/xen/common/memory.c +++ b/xen/common/memory.c @@ -33,7 +33,7 @@ * physical page frame by a domain, including uses as a page directory, * a page table, or simple mappings via a PTE. This count prevents a * domain from releasing a frame back to the hypervisor's free pool when - * it is still referencing it! + * it still holds a reference to it. * * TYPE_COUNT is more subtle. A frame can be put to one of three * mutually-exclusive uses: it might be used as a page directory, or a @@ -140,7 +140,9 @@ #include #if 0 -#define MEM_LOG(_f, _a...) printk("DOM%d: (file=memory.c, line=%d) " _f "\n", current->domain, __LINE__, ## _a ) +#define MEM_LOG(_f, _a...) + printk("DOM%d: (file=memory.c, line=%d) " _f "\n", \ + current->domain, __LINE__, ## _a ) #else #define MEM_LOG(_f, _a...) ((void)0) #endif @@ -230,7 +232,7 @@ static void __invalidate_shadow_ldt(struct task_struct *p) page = frame_table + pfn; ASSERT((page->flags & PG_type_mask) == PGT_ldt_page); ASSERT((page->flags & PG_domain_mask) == p->domain); - ASSERT((page->type_count != 0) && (page->tot_count != 0)); + ASSERT((page_type_count(page) != 0) && (page_tot_count(page) != 0)); put_page_type(page); put_page_tot(page); } @@ -271,7 +273,7 @@ int map_ldt_shadow_page(unsigned int off) page = frame_table + (l1e >> PAGE_SHIFT); if ( unlikely((page->flags & PG_type_mask) != PGT_ldt_page) ) { - if ( unlikely(page->type_count != 0) ) + if ( unlikely(page_type_count(page) != 0) ) goto out; /* Check all potential LDT entries in the page. */ @@ -367,7 +369,7 @@ static int dec_page_refcnt(unsigned long page_nr, unsigned int type) type); return -1; } - ASSERT((page_type_count(page) & ~REFCNT_PIN_BIT) != 0); + ASSERT(page_type_count(page) != 0); put_page_tot(page); return put_page_type(page); } @@ -607,7 +609,7 @@ static void put_page(unsigned long page_nr, int writeable) page = frame_table + page_nr; ASSERT(DOMAIN_OKAY(page->flags)); ASSERT((!writeable) || - (((page_type_count(page) & ~REFCNT_PIN_BIT) != 0) && + ((page_type_count(page) != 0) && ((page->flags & PG_type_mask) == PGT_writeable_page) && ((page->flags & PG_need_flush) == PG_need_flush))); if ( writeable ) @@ -735,7 +737,7 @@ static int do_extended_command(unsigned long ptr, unsigned long val) switch ( cmd ) { case MMUEXT_PIN_L1_TABLE: - if ( unlikely(page->type_count & REFCNT_PIN_BIT) ) + if ( unlikely(page->flags & PG_guest_pinned) ) { MEM_LOG("Pfn %08lx already pinned", pfn); err = 1; @@ -745,7 +747,7 @@ static int do_extended_command(unsigned long ptr, unsigned long val) goto mark_as_pinned; case MMUEXT_PIN_L2_TABLE: - if ( unlikely(page->type_count & REFCNT_PIN_BIT) ) + if ( unlikely(page->flags & PG_guest_pinned) ) { MEM_LOG("Pfn %08lx already pinned", pfn); err = 1; @@ -759,25 +761,19 @@ static int do_extended_command(unsigned long ptr, unsigned long val) MEM_LOG("Error while pinning pfn %08lx", pfn); break; } - put_page_type(page); - put_page_tot(page); - page->type_count |= REFCNT_PIN_BIT; - page->tot_count |= REFCNT_PIN_BIT; + page->flags |= PG_guest_pinned; break; case MMUEXT_UNPIN_TABLE: - if ( !DOMAIN_OKAY(page->flags) ) + if ( unlikely(!DOMAIN_OKAY(page->flags)) ) { err = 1; MEM_LOG("Page %08lx bad domain (dom=%ld)", ptr, page->flags & PG_domain_mask); } - else if ( (page->type_count & REFCNT_PIN_BIT) ) + else if ( likely(page->flags & PG_guest_pinned) ) { - page->type_count &= ~REFCNT_PIN_BIT; - page->tot_count &= ~REFCNT_PIN_BIT; - get_page_type(page); - get_page_tot(page); + page->flags &= ~PG_guest_pinned; ((page->flags & PG_type_mask) == PGT_l1_page_table) ? put_l1_table(pfn) : put_l2_table(pfn); } @@ -916,7 +912,7 @@ int do_mmu_update(mmu_update_t *ureqs, int count) mk_l2_pgentry(req.val)); break; default: - if ( page->type_count == 0 ) + if ( page_type_count(page) == 0 ) { *(unsigned long *)req.ptr = req.val; err = 0; diff --git a/xen/drivers/block/xen_block.c b/xen/drivers/block/xen_block.c index dc7a790408..78ba3c86b9 100644 --- a/xen/drivers/block/xen_block.c +++ b/xen/drivers/block/xen_block.c @@ -350,7 +350,7 @@ static int __buffer_is_valid(struct task_struct *p, /* If reading into the frame, the frame must be writeable. */ if ( writeable_buffer && ((page->flags & PG_type_mask) != PGT_writeable_page) && - (page->type_count != 0) ) + (page_type_count(page) != 0) ) { DPRINTK("non-writeable page passed for block read\n"); goto out; @@ -376,7 +376,7 @@ static void __lock_buffer(unsigned long buffer, page = frame_table + pfn; if ( writeable_buffer ) { - if ( page->type_count == 0 ) + if ( page_type_count(page) == 0 ) { page->flags &= ~PG_type_mask; /* No need for PG_need_flush here. */ diff --git a/xen/include/xeno/mm.h b/xen/include/xeno/mm.h index d565583d6a..8f0c032367 100644 --- a/xen/include/xeno/mm.h +++ b/xen/include/xeno/mm.h @@ -59,13 +59,6 @@ typedef struct pfn_info { unsigned long type_count; /* pagetable/dir, or domain-writeable refs. */ } frame_table_t; -/* - * We use a high bit to indicate that a page is pinned. - * We do not use the top bit as that would mean that we'd get confused with - * -ve error numbers in some places in common/memory.c. - */ -#define REFCNT_PIN_BIT 0x40000000UL - #define get_page_tot(p) ((p)->tot_count++) #define put_page_tot(p) \ ({ ASSERT((p)->tot_count != 0); --(p)->tot_count; }) @@ -83,9 +76,9 @@ typedef struct pfn_info { #define PG_slab 24 /* domain flags (domain != 0) */ /* - * NB. The following three flags are MUTUALLY EXCLUSIVE! + * NB. The following page types are MUTUALLY EXCLUSIVE. * At most one can be true at any point, and 'type_count' counts how many - * references exist of teh current type. A change in type can only occur + * references exist of the current type. A change in type can only occur * when type_count == 0. */ #define PG_type_mask (15<<24) /* bits 24-27 */ @@ -111,6 +104,13 @@ typedef struct pfn_info { */ #define PG_need_flush (1<<28) +/* + * This bit indicates that the guest OS has pinned the page to its current + * type. For page tables this can avoid the frame scanning and reference-count + * updates that occur when the type count falls to zero. + */ +#define PG_guest_pinned (1<<29) + #define PageSlab(page) test_bit(PG_slab, &(page)->flags) #define PageSetSlab(page) set_bit(PG_slab, &(page)->flags) #define PageClearSlab(page) clear_bit(PG_slab, &(page)->flags) @@ -118,11 +118,16 @@ typedef struct pfn_info { #define SHARE_PFN_WITH_DOMAIN(_pfn, _dom) \ do { \ (_pfn)->flags = (_dom) | PGT_writeable_page | PG_need_flush; \ - (_pfn)->tot_count = (_pfn)->type_count = 2; \ + set_page_tot_count((_pfn), 2); \ + set_page_type_count((_pfn), 2); \ } while ( 0 ) -#define UNSHARE_PFN(_pfn) \ - (_pfn)->flags = (_pfn)->type_count = (_pfn)->tot_count = 0 +#define UNSHARE_PFN(_pfn) \ + do { \ + (_pfn)->flags = 0; \ + set_page_tot_count((_pfn), 0); \ + set_page_type_count((_pfn), 0); \ + } while ( 0 ) /* The array of struct pfn_info, * free pfn list and number of free pfns in the free list diff --git a/xen/net/dev.c b/xen/net/dev.c index e7b9f2d01c..280db4def1 100644 --- a/xen/net/dev.c +++ b/xen/net/dev.c @@ -550,7 +550,8 @@ void deliver_packet(struct sk_buff *skb, net_vif_t *vif) } /* Give the new page to the domain, marking it writeable. */ - new_page->tot_count = new_page->type_count = 1; + set_page_type_count(new_page, 1); + set_page_tot_count(new_page, 1); new_page->flags = vif->domain->domain | PGT_writeable_page | PG_need_flush; list_add(&new_page->list, &vif->domain->pg_head); @@ -2113,10 +2114,10 @@ static long get_bufs_from_vif(net_vif_t *vif) if ( ((buf_page->flags & (PG_type_mask | PG_domain_mask)) != (PGT_writeable_page | p->domain)) || - (buf_page->tot_count != 1) ) + (page_tot_count(buf_page) != 1) ) { DPRINTK("Need a mapped-once writeable page (%ld/%ld/%08lx)\n", - buf_page->type_count, buf_page->tot_count, + page_type_count(buf_page), page_tot_count(buf_page), buf_page->flags); make_rx_response(vif, rx.id, 0, RING_STATUS_BAD_PAGE, 0); goto rx_unmap_and_continue; @@ -2129,7 +2130,9 @@ static long get_bufs_from_vif(net_vif_t *vif) get_page_type(pte_page); get_page_tot(pte_page); *ptep &= ~_PAGE_PRESENT; - buf_page->flags = buf_page->type_count = buf_page->tot_count = 0; + buf_page->flags = 0; + set_page_type_count(buf_page, 0); + set_page_tot_count(buf_page, 0); list_del(&buf_page->list); vif->rx_shadow_ring[j].id = rx.id; @@ -2198,7 +2201,8 @@ long flush_bufs_for_vif(net_vif_t *vif) *pte = (rx->buf_pfn<flags |= PGT_writeable_page | PG_need_flush; - page->type_count = page->tot_count = 1; + set_page_type_count(page, 1); + set_page_tot_count(page, 1); } unmap_domain_mem(pte); diff --git a/xen/net/skbuff.c b/xen/net/skbuff.c index 4e07cb9382..d8950633b9 100644 --- a/xen/net/skbuff.c +++ b/xen/net/skbuff.c @@ -162,7 +162,9 @@ static inline void dealloc_skb_data_page(struct sk_buff *skb) spin_lock_irqsave(&free_list_lock, flags); - pf->flags = pf->type_count = pf->tot_count = 0; + pf->flags = 0; + set_page_type_count(pf, 0); + set_page_tot_count(pf, 0); list_add(&pf->list, &free_list); free_pfns++; -- 2.30.2